#!/usr/bin/env python

'''Tool to assemble and run the RQL language tests, including the polyglot yaml tests'''

from __future__ import print_function

import atexit, collections, copy, datetime, distutils.version, distutils.spawn, itertools, json, optparse, os
import re, resource, shutil, signal, stat, string, subprocess, sys, tempfile, threading, time, traceback, warnings

sys.path.insert(0, os.path.join(os.path.dirname(os.path.realpath(__file__)), os.pardir, "common"))
import driver, parsePolyglot, test_exceptions, utils, http_support

try:
    unicode
except NameError:
    unicode = str
try:
    import Queue
except ImportError:
    import queue as Queue
try:
    import reprlib
except ImportError:
    import repr as reprlib

# == constants

TIMED_OUT_EXIT_CODE = -257 # outside the range of exit and signal codes
CANCELED_EXIT_CODE  = -278

# -- repr subclass

class ReprReplace(reprlib.Repr):
    def repr_str(self, obj, level):
        result = repr(obj)
        if result.startswith("'"):
            result = '"' + result[1:-1].replace('"', '\\"') + '"'
        return result
    
    def repr_unicode(self, obj, level):
        result = self.repr_str(obj, level)
        if len(result) > 1 and result[:2] in ('u"', "u'"):
            result = result[1:]
        return result
replaceRepr = ReprReplace()

# -- ensure we can open enough files

try:
    openFileLimit = int(os.getenv('OPEN_FILE_LIMIT', 512))
    if resource.getrlimit(resource.RLIMIT_NOFILE)[0] < openFileLimit:
        resource.setrlimit(resource.RLIMIT_NOFILE, (openFileLimit, openFileLimit))
except Exception as e:
    sys.exit('Bad OPEN_FILE_LIMIT input: %s' % os.getenv('OPEN_FILE_LIMIT'))

# -- settings

testExtensions = ['httpbin', 'mocha', 'test', 'yaml']

r = None

print_debug = False

# -- internal global variables

testLock = threading.Lock()
devNull = open(os.devnull)

# -- signal and atexit handlers

cancelRun = False
def setCancelRun(*args):
    global cancelRun
    cancelRun = True
    print('') # put the ^C on a separate line
signal.signal(signal.SIGINT, setCancelRun)
# ToDo: cover all relevant signals
# ToDo: allow for double-SIGINT to fast-kill

# --

def debug(message):
    if print_debug:
        sys.stdout.write('DEBUG: %s' % message.rstrip())

# --

# This class attempts to abstract some of the details that separate our source languages.
class SrcLang(object):
    test_types = None
    language_name = None
    display_name = None
    
    interpreter_name_exe_names = None
    interpreter_version = None
    interpreter_env_name = None
    
    driver_info = None
    
    envVariablesToSet = None
    
    # yaml translation constants
    
    none_string = 'None'
    line_comment = '#'
    lang_replaces = {}
    lang_regex = None # set in __new__
    
    # internal caches
    
    _interpreter_path = None
    __polyglot_language_header = None
    _polyglot_language_header_path = None
    
    instance_setup = False
    
    # -- class variables
    
    _singletons = {}
    
    def __new__(cls, version=''):
        
        if cls.lang_regex is None:
            cls.lang_regex = re.compile(r'''(?P<quoteChar>["']?)\b(?P<target>%s)\b(?P=quoteChar)''' % '|'.join(cls.lang_replaces.keys()))
        
        singletonName = cls.__name__
        if version not in (None, ''):
            singletonName = "%s-%s" % (cls.__name__, str(version))
        if singletonName not in cls._singletons:
            cls._singletons[singletonName] = super(SrcLang, cls).__new__(cls)
        return cls._singletons[singletonName]
    
    def __init__(self, version=None):
        assert self.language_name is not None, 'SrcLang subclass is not properly setup'
        assert self.language_name in utils.driverPaths, 'SrcLang %s info not in utils.driverPaths' % self.language_name # ToDo: keep all of this in one file
        
        self.envVariablesToSet = {}
        self.class_setup(version)
        self.instance_setup = True
        self.driver_info = utils.driverPaths[self.language_name]
        try:
            self.envVariablesToSet['INTERPRETER_PATH'] = self.interpreter_path
        except Exception: pass
    
    # Translates names from canonical name representation
    # (underscores) to the convention for this language
    @staticmethod
    def nametranslate(name):
        return name.replace('__', '_')
    
    # Translates dictionary (object) representation from canonical
    # form (e.g. "{'a':1}") to the appropriate representation for this
    # language
    @staticmethod
    def dict_translate(dic):
        return dic
    
    @classmethod
    def term_translate(cls, content):
        '''translate language-specific terms, unless alone in quotes or part of a word'''
        
        def termTranslateFunction(match):
            if match.group('quoteChar') in (None, ''):
                return cls.lang_replaces.get(match.group('target')) or match.group('target')
            else:
                return match.group()
        return cls.lang_regex.sub(termTranslateFunction, content)
    
    # Translate a generic code string using the rules defined by `self`
    def translate_query(self, src, quote=True, name=False):
        result = self.dict_translate(self.term_translate(unicode(src).strip()))
        if name:
            result = self.nametranslate(result)
        if quote:
            result = replaceRepr.repr(result)
        return result
    
    @property
    def interpreter_path(self):
        '''return the path to the interpreter on this system, respecting the given env variable'''
        
        if self._interpreter_path is not None:
            return self._interpreter_path
        
        if self.interpreter_name_exe_names in (None, []):
            raise NotImplementedError('interpreter_name_exe_names is not implemented as required for the SrcLang subclass: %s' % self.__class__.__name__)
        if self.interpreter_env_name is None:
            raise NotImplementedError('interpreter_env_name is not implemented as required for the SrcLang subclass: %s' % self.__class__.__name__)
        
        executablePath = os.getenv(self.interpreter_env_name)
        if executablePath is None:
            for name in self.interpreter_name_exe_names:
                candidatePath = distutils.spawn.find_executable(name)
                if candidatePath is not None:
                    try:
                        self.interpreter_version = self.version_check(candidatePath)
                        executablePath = candidatePath
                        break
                    except Exception:
                        pass
        else:
            self.interpreter_version = self.version_check(executablePath) # exception on failure
            
        if executablePath is None:
            raise test_exceptions.TestingFrameworkException(detail='Unable to find interpreter %s' % self.display_name)
        
        if not os.access(executablePath, os.X_OK):
            raise test_exceptions.TestingFrameworkException(detail='The interpreter %s is not executable' % executablePath)
        
        self._interpreter_path = executablePath
        return self._interpreter_path
    
    def get_interpreter_version(self, executablePath):
        '''Return the version string for this interpreter, this must be implemented by subclasses'''
        raise NotImplementedError('version_check is not implemented as required for the SrcLang subclass: %s' % self.__class__.__name__)
    
    def version_check(self, executablePath):
        '''Check that this is a supported version of this interpreter'''
        
        targetVersion = self.get_interpreter_version(executablePath)
        
        if self.minVersion is not None and targetVersion < self.minVersion:
            raise test_exceptions.TestingFrameworkException(detail='Version of %s (%s) was below the minimum version %s' % (executablePath, targetVersion, self.minVersion))
        
        if self.maxVersion is not None and targetVersion > self.maxVersion:
            raise test_exceptions.TestingFrameworkException(detail='Version of %s (%s) was above the maximum version %s for python2' % (executablePath, targetVersion, self.maxVersion))
        
        return targetVersion
    
    @property
    def polyglot_language_header(self):
        if self.__polyglot_language_header is None:
            with open(self._polyglot_language_header_path, 'rU') as headerFile:
                self.__class__.__polyglot_language_header = headerFile.read().decode('utf-8')
        return self.__polyglot_language_header

class PyLang(SrcLang):
    language_name = 'Python'
    
    interpreter_name_exe_names = None
    interpreter_env_name = 'PYTHON'
    minVersion = None
    maxVersion = None
    
    _polyglot_language_header_path = os.path.join(os.path.dirname(__file__), 'drivers', 'driver.py')
    
    lang_replaces = {
        'null': 'None',
        'nil': 'None'
    }
    
    def class_setup(self, version):
        if self.instance_setup is True:
            return
        
        self.display_name = self.language_name + ' ' + version
        self.test_types = []
        self.interpreter_name_exe_names = []
        
        self.minVersion = distutils.version.LooseVersion(version)
        self.maxVersion = distutils.version.LooseVersion(version + '.999')
        
        if version != '':
            lastVersion = None
            workingVersion = version
            while lastVersion != workingVersion:
                self.test_types.append('py' + workingVersion)
                self.interpreter_name_exe_names.append('python' + workingVersion)
                
                lastVersion = workingVersion
                workingVersion = os.path.splitext(workingVersion)[0]
        self.test_types.append('py')
        self.interpreter_name_exe_names.append('python')
        self.interpreter_name_exe_names.reverse() # make `python` the first checked so virtualenv can work
    
    def get_interpreter_version(self, executablePath):
        versionCheckProcess = subprocess.Popen([executablePath, '--version'], stdin=devNull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
        output, __ = versionCheckProcess.communicate()
        
        if versionCheckProcess.returncode != 0:
            raise test_exceptions.TestingFrameworkException(detail='Unable to determine the version of %s' % executablePath)
        
        versionNumbers = re.findall(r'\b([\d\.]+)\b', str(output.decode('utf-8')))
        if len(versionNumbers) != 1:
            raise test_exceptions.TestingFrameworkException(detail='Got multiple possible version numbers for %s' % executablePath)
        
        return distutils.version.LooseVersion(str(versionNumbers[0]))

class JsLang(SrcLang):
    language_name = 'JavaScript'
    
    interpreter_name_exe_names = ['node']
    interpreter_env_name = 'NODE'
    minVersion = None # no version requirements yet
    maxVersion = None
    
    _polyglot_language_header_path = os.path.join(os.path.dirname(__file__), 'drivers', 'driver.js')
    
    lang_replaces = {
        'None': 'null',
        'nil': 'null',
        'True': 'true',
        'False': 'false'
    }
    line_comment = '//'
    
    nametranslate_regex = re.compile(r"(?P<quoteChar>[\"\'\`]?)\$?\b[a-z$]+(_[a-z$]+)+\b\$?(?P=quoteChar)")
    
    def class_setup(self, version):
        if self.instance_setup is True:
            return
        
        self.display_name = self.language_name + ' ' + version
        self.test_types = []
        self.interpreter_name_exe_names = []
        
        self.minVersion = distutils.version.LooseVersion(version)
        self.maxVersion = distutils.version.LooseVersion(version + '.999')
        
        if version != '':
            lastVersion = None
            workingVersion = version
            while lastVersion != workingVersion:
                self.test_types.append('js' + workingVersion)
                self.interpreter_name_exe_names.append('node' + workingVersion)
                
                lastVersion = workingVersion
                workingVersion = os.path.splitext(workingVersion)[0]
        self.test_types.append('js')
        self.interpreter_name_exe_names.append('node')
    
    # Converts canonical form (underscore separation) to camel case
    @classmethod
    def nametranslate(cls, name):
        def replaceFnct(match):
            if match.group('quoteChar') not in (None, ''):
                return match.group()
            returnStr = ''
            upCase = False
            for char in match.group():
                if upCase:
                    upCase = False
                    returnStr += char.upper()
                else:
                    if char == '_':
                        upCase = True
                    else:
                        upCase = False
                        returnStr += char
            return returnStr
        
        name = re.sub(cls.nametranslate_regex, replaceFnct, name)
        return re.sub('__', '_', name)
    
    def translate_query(self, src, quote=True, name=False):
        '''Handle the errors from `eval({a:1})`: JavaScript thinks it is a code block'''
        result = super(JsLang, self).translate_query(src, quote=quote, name=name)
        if quote and re.match(r'["\']\{', result):
            result = result[0] + '(' + result[1:-1] + ')' + result[-1]
        return result
    
    def get_interpreter_version(self, executablePath):
        versionCheckProcess = subprocess.Popen([executablePath, '--version'], stdin=devNull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
        output, __ = versionCheckProcess.communicate()
        
        if versionCheckProcess.returncode != 0:
            raise test_exceptions.TestingFrameworkException(detail='Unable to determine the version of %s' % executablePath)
        
        versionNumber = str(output.decode('utf-8')).lstrip('v').strip()
        return distutils.version.LooseVersion(versionNumber)

class RbLang(SrcLang):
    language_name = 'Ruby'
    
    interpreter_name_exe_names = None
    interpreter_env_name = 'RUBY'
    minVersion = None
    maxVersion = None
    
    lang_replaces = {
        'None': 'nil',
        'null': 'nil',
        'True': 'true',
        'False': 'false'
    }
    
    _polyglot_language_header_path = os.path.join(os.path.dirname(__file__), 'drivers', 'driver.rb')
    
    def class_setup(self, version):
        if self.instance_setup is True:
            return
        
        self.display_name = self.language_name + ' ' + version
        self.test_types = []
        self.interpreter_name_exe_names = []
        
        self.minVersion = distutils.version.LooseVersion(version)
        self.maxVersion = distutils.version.LooseVersion(version + '.999')
        
        if version != '':
            lastVersion = None
            workingVersion = version
            while lastVersion != workingVersion:
                self.test_types.append('rb' + workingVersion)
                self.interpreter_name_exe_names.append('ruby' + workingVersion)
                
                lastVersion = workingVersion
                workingVersion = os.path.splitext(workingVersion)[0]
        self.test_types.append('rb')
        self.interpreter_name_exe_names.append('ruby')

    @classmethod
    def dict_translate(cls, source):
        '''Given a string, find things that look like Python-esque dicts and translate them to ruby syntax'''
        
        levelChars = {
            '}':'{',
            ']':'[',
            ')':'(',
            '"':'"',
            "'":"'"
        }
        
        stateStack = []
        escaped = False
        source = list(unicode(source))
        length = len(source)
        for pos, char in enumerate(source):
            
            # in a quote
            if len(stateStack) > 0 and stateStack[-1] in ('"', "'"):
                if escaped is True:
                    escaped = False
                elif char == '\\':
                    escaped = True
                elif char == stateStack[-1]:
                    stateStack.pop()
            
            # start of level
            elif char in levelChars.values():
                stateStack.append(char)
            
            # skip everything not in a level
            elif len(stateStack) == 0:
                pass
            
            # end of non-dict level
            elif char in levelChars.keys() and stateStack[-1] == levelChars[char]:
                stateStack.pop()
            
            # candidate points
            elif char == ':' and stateStack[-1] == '{':
                
                # check that to the right and left look good
                validPreChars = tuple(string.letters)
                validPostChars = validPreChars + tuple(string.digits) + tuple(levelChars.values()) + ('+', '-')
                try:
                    # look at the preceding characters to see if this looks valid
                    startedWord = False
                    for i in xrange(pos - 1, -1, -1):
                        checkChar = source[i]
                        if checkChar in string.whitespace:
                            if startedWord:
                                break
                            continue
                        elif checkChar in string.digits: # {a0:4} is valid, {5:4} is not
                            startedWord = True
                            continue
                        elif checkChar in levelChars.keys():
                            if startedWord:
                                raise AssertionError() # too complicated, probably not a dict key
                            break # either a quotes string, a dict, or an array, all are valid
                        elif i >= len('nil') and ''.join(source[pos - len('nil'): pos]) == 'nil':
                            if i > len('nil') + 1:
                                if source[pos - len('nil') - 1] in tuple(string.letters) + tuple(string.digits) + tuple('_'):
                                    startedWord = True
                                    continue
                                else:
                                    break # we need the => to keep it from auto-translating {nil:1} to {:nil:1}
                            break
                        elif checkChar in validPreChars:
                            raise AssertionError() # these are auto-translated by Ruby 1.9+: {a:1} => {:a=>1}
                        else:
                            if startedWord:
                                break
                            raise AssertionError()
                    else:
                        if not(startedWord) and i != 0:
                            raise AssertionError()
                    
                    # look at the following characters to make sure that looks good as well
                    for i in xrange(pos + 1, length):
                        checkChar = source[i]
                        if checkChar in string.whitespace:
                            continue
                        elif checkChar in validPostChars:
                            break
                        else:
                            raise AssertionError()
                    else:
                        raise AssertionError()
                    
                    # passed both tests, replace it
                    source[pos] = '=>'
                    
                except AssertionError: pass
        
        # - return the string, with extraneous newlines striped out
        
        return ''.join((x.strip('\n') for x in source))
    
    def get_interpreter_version(self, executablePath):
        '''Return the version string for this interpreter'''
        versionCheckProcess = subprocess.Popen([executablePath, '--version'], stdin=devNull, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, close_fds=True)
        output, __ = versionCheckProcess.communicate()
        
        if versionCheckProcess.returncode != 0:
            raise test_exceptions.TestingFrameworkException(detail='Unable to determine the version of %s' % executablePath)
        
        versionNumber = str(output.decode('utf-8').split()[1]).strip()
        return distutils.version.LooseVersion(versionNumber)

class JrbLang(RbLang):
    language_name = 'JRuby'
    
    def class_setup(self, version):
        if self.instance_setup is True:
            return
        
        self.display_name = self.language_name + ' ' + version
        self.test_types = []
        self.interpreter_name_exe_names = []
        
        self.minVersion = distutils.version.LooseVersion(version)
        self.maxVersion = distutils.version.LooseVersion(version + '.999')
        
        if version != '':
            lastVersion = None
            workingVersion = version
            while lastVersion != workingVersion:
                self.test_types.append('jrb' + workingVersion)
                self.interpreter_name_exe_names.append('jruby' + workingVersion)
                
                lastVersion = workingVersion
                workingVersion = os.path.splitext(workingVersion)[0]
        self.test_types.append('jrb')
        self.test_types.append('rb')
        self.interpreter_name_exe_names.append('jruby')

interpreters = {
    'js':     [JsLang('0.10'), JsLang('0.12'), JsLang('4.0'), JsLang('4.1'), JsLang('4.2'), JsLang('5.0'), JsLang('5.1'), JsLang('5.2'), JsLang('5.3')],
    'py':     [PyLang('2.6'), PyLang('2.7'), PyLang('3.0'), PyLang('3.1'), PyLang('3.2'), PyLang('3.3'), PyLang('3.4'), PyLang('3.5')],
    'rb':     [RbLang('1.9'), RbLang('2.0'), RbLang('2.1'), RbLang('2.3'), JrbLang('9.0')],
    'jrb':    [JrbLang('9.0')],
    
    'js0':    [JsLang('0.10'), JsLang('0.12')],
    'js4':    [JsLang('4.0'), JsLang('4.1'), JsLang('4.2')],
    'js5':    [JsLang('5.0'), JsLang('5.1'), JsLang('5.2'), JsLang('5.3')],
    
    'py2':    [PyLang('2.6'), PyLang('2.7')],
    'py2.6':  [PyLang('2.6')],
    'py2.7':  [PyLang('2.7')],
    
    'py3':    [PyLang('3.0'), PyLang('3.1'), PyLang('3.2'), PyLang('3.3'), PyLang('3.4'), PyLang('3.5')],
    'py3.0':  [PyLang('3.0')],
    'py3.1':  [PyLang('3.1')],
    'py3.2':  [PyLang('3.2')],
    'py3.3':  [PyLang('3.3')],
    'py3.4':  [PyLang('3.4')],
    'py3.5':  [PyLang('3.5')],
    
    'rb1':    [RbLang('1.9')],
    'rb1.9':  [RbLang('1.9')],
    
    'rb2':    [RbLang('2.0'), RbLang('2.1'), RbLang('2.2')],
    'rb2.0':  [RbLang('2.0')],
    'rb2.1':  [RbLang('2.1')],
    'rb2.2':  [RbLang('2.2')],
    
    'jrb9.0': [JrbLang('9.0')]
}

# Abstracts a set of tests given in a single file
class TestGroup(object):
    
    testLanguageEntry = collections.namedtuple('testLanguageEntry', ['command', 'expected', 'definition', 'runopts', 'testopts'])
    variableRegex = re.compile(r'^\s*(?P<quoteChar>[\'\"]?)\s*(?P<variableName>[a-zA-Z][\w\[\]\{\}\'\"]*)\s*=\s*(?P<expression>[^=].+)(?P=quoteChar)\s*$', flags=re.DOTALL | re.MULTILINE)
    
    @classmethod
    def buildYamlTest(cls, testName, sourceFile, lang, outputPath, shards=1, useSpecificTable=False):
        # -- input validation
        
        # testName
        if testName is None:
            raise ValueError('buildYamlTest requires a testName, got None')
        testName = str(testName)
        
        # sourceFile
        if not sourceFile or not (hasattr(sourceFile, 'read') or os.path.isfile(sourceFile)):
            raise ValueError('buildYamlTest requires a sourceFile, got: %r' % sourceFile)
        
        # lang
        if lang is None:
            raise ValueError('buildYamlTest requires a lang, got None')
        if not isinstance(lang, SrcLang):
            raise ValueError('buildYamlTest requires a subclass of SrcLang, got %s' % lang)
        
        # outputPath
        if outputPath is None:
            raise ValueError('buildYamlTest requires an outputPath, got None')
        if not os.path.isdir(os.path.dirname(outputPath)):
            if not os.path.exists(os.path.dirname(outputPath)):
                os.makedirs(os.path.dirname(outputPath))
            else:
                raise ValueError('buildYamlTest got an outputPath with a directory that was already a non-file: %s' % outputPath)
        if os.path.exists(outputPath) and not os.path.isfile(outputPath):
            raise ValueError('buildYamlTest got an outputPath that was already a non-file: %s' % outputPath)
        
        # -- create the file
        
        parsed_data = None
        try:
            parsed_data = parsePolyglot.parseYAML(sourceFile)
        except Exception as e:
            raise ValueError('buildYamlTest got a sourceFile (%s) that was unable to be parsed as Yaml: %s' % (sourceFile, str(e)))
        if parsed_data is None:
            raise ValueError('buildYamlTest got a sourceFile (%s) that was unable to be parsed as Yaml' % sourceFile)
        if not 'tests' in parsed_data or not isinstance(parsed_data['tests'], list):
            raise ValueError('buildYamlTest got a sourceFile (%s) without any tests' % sourceFile)
        
        with open(outputPath, 'w') as outputFile:
            
            # - write the shebang line
            
            outputFile.write('#!%s\n' % lang.interpreter_path)
            
            # - write in the encoding (mostly for Python2.6)
            
            outputFile.write('%s -*- coding: utf-8 -*-\n\n' % lang.line_comment)
            
            # - mark start of the header/driver
            
            outputFile.write('%s ========== Start of common test functions ==========\n\n' % lang.line_comment)
            
            # - insert a marker telling what test this is
            
            outputFile.write('\n%s Tests %s (%s): %s (%s)\n\n' % (lang.line_comment, testName, lang.display_name, parsed_data.get('desc', ''), sourceFile))
            
            # - write the header
            
            outputFile.write(lang.polyglot_language_header)
            
            # - generate the table setup lines
            
            outputFile.write('%s -- Table Creation\n\n' % lang.line_comment)
            
            if "table_variable_name" in parsed_data:
                variableNames = parsed_data['table_variable_name']
                if hasattr(parsed_data['table_variable_name'], 'upper'):
                    variableNames = re.findall(r'([a-zA-Z\d\.\_\-]+)', parsed_data['table_variable_name'])
                
                for index, variableName in enumerate(variableNames):
                    db_name, table_name = utils.get_test_db_table(tableName=testName, index=index)
                    outputFile.write("setup_table(%s, %s, %s)\n" % (
                        lang.translate_query(variableName, name=True),
                        lang.translate_query(table_name),
                        lang.translate_query(db_name)
                    ))
                outputFile.write('\n')
            else:
                outputFile.write('%s No auto-created tables\n\n' % lang.line_comment)
            
            # - test that all external tables are claimed
            
            outputFile.write('setup_table_check()\n\n')
            
            # - mark end of the header/driver
            
            outputFile.write('%s ========== End of common test functions ==========\n\n' % lang.line_comment)
            
            # - add the body of the tests
            
            for testNumber, test in enumerate(parsed_data['tests'], start=0):
                cls.write_test(testName, outputFile, test, lang, str(testNumber), shards=shards)
            
            outputFile.write('\n\n')
            
            # - add the footer
            
            outputFile.write('the_end()\n')
            
        # - make sure the file is executable
        
        os.chmod(outputPath, stat.S_IRWXU | stat.S_IRWXG | stat.S_IROTH | stat.S_IXOTH)
    
    @classmethod
    def write_test(cls, testName, out, test, lang, index, shards=1):
        
        testName = os.path.join(testName, str(index))

        # See if this test defines operations for our language
        try:
            testCode = cls.get_lang_entries(test, lang)
        except Exception as e:
            debug(traceback.format_exc())
            raise Exception("Error while processing test: %s\n%s" % (testName, str(e)))
        
        # Does this test define a variable?
        if testCode.definition:
            cls.write_def(testName, lang, out, testCode.definition)
        
        # Write the commands
        if testCode.command:
            for test_case in testCode.command:
                
                variableMatch = cls.variableRegex.match(test_case)
                if variableMatch:
                    testCode.testopts['variable'] = parsePolyglot.yamlValue(lang.nametranslate(variableMatch.group('variableName')))
                    if isinstance(test_case, parsePolyglot.yamlValue):
                        test_case = parsePolyglot.yamlValue(variableMatch.group('expression'), test_case.linenumber)
                    else:
                        test_case = variableMatch.group('expression')
                
                try:
                    out.write("test(%(code)s, %(expected)s, %(name)s, %(runopts)s, %(testopts)s)\n" % {
                        'code': lang.translate_query(test_case, name=True),
                        'expected': '"anything()"' if testCode.expected is None else lang.translate_query(testCode.expected),
                        'name': '"line %d"' % test_case.linenumber if hasattr(test_case, 'linenumber') else repr(testName),
                        'runopts': lang.translate_query(testCode.runopts, quote=False),
                        'testopts': lang.translate_query(testCode.testopts, quote=False)
                    })
                    
                    # We want to auto-shard tables that we create. There is still no
                    # way to do this from ReQL so we have to hack the admin cli.
                    
                    if shards > 1:
                        pattern = 'table_create\\(\'(\\w+)\'\\)'
                        mo = re.search(pattern, test_case)
                        if mo:
                            table_name = mo.group(1)
                            out.write('shard("%s")\n' % table_name)
                except Exception as e:
                    debug(traceback.format_exc())
                    raise Exception("Error while processing test: %s\n%s\n%s" % (testName, str(testCode.command), str(e)))

    @classmethod
    def write_def(cls, testName, lang, out, defs):
        for definition in defs:
            variableString = ''
            
            # parse the variable name
            
            variableMatch = cls.variableRegex.match(definition)
            if not variableMatch:
                raise Exception('Error while processing test %s: def entry missing variable name: %s' % (testName, str(definition)))
            
            variableString = lang.translate_query(variableMatch.group('variableName'), name=True)
            definition = lang.translate_query(variableMatch.group('expression'), name=True)
            
            # write the output
            
            out.write('define(%s, %s)\n' % (definition, variableString))
    
    # Tests may specify generic test strings valid for all languages or language specific versions
    @classmethod
    def get_lang_entries(cls, source, lang, default='command'):
        assert default in ('command', 'expected', 'definition'), 'get_lang_entries requires that default be one of : command, expected, or definition'
        
        results = {'command':None, 'definition':None, 'expected':None, 'runopts':{}, 'testopts':{}}
        
        if source is None:
            pass
        
        elif default in ('runopts', 'testopts'):
            results[default] = source
        
        elif not hasattr(source, 'get'):
            # leaf, replace appropriate item
            if default != 'expected' and hasattr(source, '__iter__') and not hasattr(source, 'capitalize'):
                results[default] = []
                for item in source:
                    results[default].append(item)
            else:
                if default == 'expected':
                    results['expected'] = source
                else:
                    results[default] = [source]
        else:
            if default == 'command':
                
                # runopts and testopts
                if 'runopts' in source:
                    results['runopts'].update(source['runopts'])
                if 'testopts' in source:
                    results['testopts'].update(source['testopts'])
                
                # def and ot
                if 'def' in source:
                    results = cls.combineLangEntries(results, cls.get_lang_entries(source['def'], lang, 'definition'))
                if 'ot' in source:
                    results = cls.combineLangEntries(results, cls.get_lang_entries(source['ot'], lang, 'expected'))
                
                # command
                for langCode in lang.test_types + ['cd']:
                    if langCode in source:
                        results = cls.combineLangEntries(results, cls.get_lang_entries(source[langCode], lang, 'command'))
                        break
                    
            elif default == 'expected':
                for langCode in lang.test_types + ['cd']:
                    if langCode in source:
                        results['expected'] = cls.get_lang_entries(source[langCode], lang, 'expected').expected
                        break
            
            elif default == 'definition':
                for langCode in lang.test_types + ['cd']:
                    if langCode in source:
                        results['definition'] = cls.get_lang_entries(source[langCode], lang, 'definition').definition
                        break
            
            else:
                sys.stderr.write('Warning! Unknown default type: %s\n' % str(default))
        
        return cls.testLanguageEntry(command=results['command'], expected=results['expected'], definition=results['definition'], runopts=results['runopts'], testopts=results['testopts'])
    
    @classmethod
    def combineLangEntries(cls, left, right):
        
        # - convert both to dicts
        
        if isinstance(left, cls.testLanguageEntry):
            left = dict(left._asdict())
        elif not isinstance(left, dict):
            raise ValueError('combineLangEntries given a left that is not useable: %s' % str(left))
        
        if isinstance(right, cls.testLanguageEntry):
            right = dict(right._asdict())
        elif not isinstance(right, dict):
            raise ValueError('combineLangEntries given a right that is not useable: %s' % str(right))
        
        # combine them, keeping any non-None entries from right
        returnDict = {}
        for key in set(right.keys() + left.keys()): # should be the same, but...
            val = right.get(key)
            if val in (None, [], {}):
                val = left.get(key)
            if key in ('runopts', 'testopts') and val is None:
                val = {}
            returnDict[key] = val
        
        return returnDict

class Server(object):
    '''Wrapper class for a RethinkdbServer, by default this will spin up a new instance'''
    
    __conn   = None
    __server = None
    
    connectedServerIds  = None
    existingDBsToTables = None
    existingUsers       = None
    existingPermissions = None
    
    def __init__(self, *args, **kwargs):
        
        # - setup subclass
        self.subclass_init(*args, **kwargs)
        
        # - collect list of existing tables
        self.existingDBsToTables = {}
        for dbName in r.db_list().run(self.conn):
            if dbName == 'rethinkdb':
                continue
            self.existingDBsToTables[dbName] = r.db(dbName).table_list().run(self.conn)
        
        # - collect list of connected servers
        self.connectedServerIds = list(r.db('rethinkdb').table('server_config')['id'].run(self.conn))
        
        # - collect list of non-default users/permissions
        self.existingUsers = dict((x['id'], x) for x in r.db('rethinkdb').table('users').run(self.conn))
        self.existingPermissions = dict((tuple(x['id']), x) for x in r.db('rethinkdb').table('permissions').run(self.conn))
    
    def subclass_init(self, scratchDir=None):
        # - start the server
        clusterFolder = tempfile.mkdtemp(prefix='cluster_', dir=scratchDir)
        self.__server = driver.Process(name=clusterFolder, console_output=True)
    
    @property
    def conn(self):
        if not self.__conn:
            self.__conn = r.connect(host=self.host, port=self.driver_port)
        return self.__conn
    
    @property
    def host(self):
        return self.__server.host
    
    @property
    def driver_port(self):
        return self.__server.driver_port
    
    @property
    def logfile_path(self):
        return self.__server.logfile_path
    
    @property
    def data_path(self):
        if self.__server and hasattr(self.__server, 'data_path'):
            return self.__server.data_path
        return None
    
    @property
    def console_output(self):
        if self.__server and hasattr(self.__server, 'console_output'):
            return self.__server.console_output
        return None
    
    def check(self):
        '''Check that the server is accessible and minimally sane'''
        assert r.expr(1).run(self.conn) == 1
    
    def clean(self, final=False):
        
        # - clean rethinkdb._debug_scratch
        r.db('rethinkdb').table('_debug_scratch').delete().run(self.conn)
        
        # - restore users/permissions
        expectedUsers = self.existingUsers
        if not final:
            expectedUsers = copy.deepcopy(self.existingUsers)
            expectedUsers.update({
                'admin':     {'id':'admin',     'password':False},
                'test_user': {'id':'test_user', 'password':False}
            })
        
        expectedPermissions = self.existingPermissions
        if not final:
            expectedPermissions = dict((key, value) for key, value in expectedPermissions.items() if value['user'] != 'admin')
            expectedPermissions.update({
                ('test_user',): {'id':['test_user'], 'permissions':{'config':True, 'connect':True, 'read':True, 'write':True}}
            })
        
        # wipe non-expected users/permissions
        res = r.db('rethinkdb').table('users').filter(lambda row: r.expr(expectedUsers.keys()).contains(row["id"]).not_()).delete().run(self.conn)
        assert res['errors'] == 0, 'Failed removing users from %s: %s' % (self.__server.name, res)
        res = r.db('rethinkdb').table('permissions').filter(r.row['user'].ne('admin')).filter(lambda row: r.expr(expectedPermissions.keys()).contains(row["id"]).not_()).delete().run(self.conn)
        assert res['errors'] == 0, 'Failed removing permissions from %s: %s' % (self.__server.name, res)
        
        # insert inital users/permissions
        existingUsers = dict((x['id'], x) for x in r.db('rethinkdb').table('users').run(self.conn))
        for username, userinfo in expectedUsers.items():
            if username in existingUsers:
                if existingUsers[username] != userinfo:
                     res = r.db('rethinkdb').table('users').get(username).update(userinfo).run(self.conn)
                     assert res['unchanged'] + res['replaced'] == 1, 'Unable to update user %s: %s' % (username, res)
            else:
                res = r.db('rethinkdb').table('users').insert(userinfo).run(self.conn)
                assert res['inserted'] == 1, 'Unable to insert user %s: %s' % (username, res)
        
        if expectedPermissions:
            existingPermissions = dict((tuple(x['id']), x) for x in r.db('rethinkdb').table('permissions').run(self.conn))
            for permissionsName, permissionsInfo in expectedPermissions.items():
                if permissionsName in existingPermissions:
                    if existingPermissions[permissionsName] != permissionsInfo:
                         res = r.db('rethinkdb').table('permissions').get(permissionsName).update(permissionsInfo).run(self.conn)
                         assert res['unchanged'] + res['replaced'] == 1, 'Unable to update user permissions %s: %s' % (username, res)
                else:
                    res = r.db('rethinkdb').table('permissions').insert(permissionsInfo).run(self.conn)
                    assert res['inserted'] == 1, 'Unable to insert user permissions %s: %s' % (permissionsName, res)
        
        # - drop any databases or tables that were not there initially
        for dbName in r.db_list().run(self.conn):
            if dbName == 'rethinkdb':
                continue
            if dbName not in self.existingDBsToTables and not (final and dbName == 'test'):
                # drop dbs that were not there inially, except the `test` databse durring testing
                r.db_drop(dbName).run(self.conn)
            else:
                # drop any tables that were not in this db initally
                r.db(dbName).table_list().set_difference(self.existingDBsToTables.get(dbName, [])).for_each(r.db(dbName).table_drop(r.row)).run(self.conn)
        
        # - remove any newly connected servers
        r.db('rethinkdb').table('server_config').filter(lambda row: r.expr(self.connectedServerIds).contains(row['id']).not_()).delete().run(self.conn)
        
        if not final:
            # - ensure that the test db is present
            r.expr(['test']).set_difference(r.db_list()).for_each(r.db_create(r.row)).run(self.conn)
        
            # - wait for the test database to be completely ready
            r.db('test').wait(wait_for='all_replicas_ready', timeout=5).run(self.conn)
    
    def stop(self):
        self.__server.stop()

class ExternalServer(Server):
    '''Used to wrap externally provided servers'''
    
    __driver_port = None
    __host        = 'localhost'
    
    def subclass_init(self, driver_port, host=None):
        assert isinstance(driver_port, int) and driver_port > 0, 'Bad value for driver_port: %r' % driver_port
        self.__driver_port = driver_port
        if host:
            self.__host = host
        
        # - set it up to make sure it gets cleaned
        atexit.register(self.clean, final=True)
    
    @property
    def host(self):
        return self.__host
    
    @property
    def driver_port(self):
        return self.__driver_port
    
    @property
    def logfile_path(self):
        return None # change after https://github.com/rethinkdb/rethinkdb/issues/4512
    
    def stop(self):
        # we don't stop the server, but we should clean up
        self.clean()

def check_language(option, opt_str, value, parser):
    if value not in option.choices:
        raise optparse.OptionValueError('Invalid language: %s' % value) 
    
    if not hasattr(parser.values, option.dest):
        setattr(parser.values, option.dest, [])
    selectedValues = getattr(parser.values, option.dest)
    if selectedValues is None:
        selectedValues = []
    
    foundAny = False
    for language in interpreters[value]:
        try:
            language.interpreter_path # errors if it is not present/found
            foundAny = True
            if language not in selectedValues:
                selectedValues.append(language)
                break # default is to only take one language from a group
        except Exception:
            continue
    
    if foundAny is False:
        raise optparse.OptionValueError('Unable to find a valid interpreter for language: %s' % value)
    
    setattr(parser.values, option.dest, selectedValues)

class Test(object):
    
    # == class/instance variables
    
    timeout = 300 # seconds to timeout, this can be changed per-test
    
    # == instance variables
    
    name = None
    type = None
    path = None
    driverLang = None
    resultDir = None # where the test ouput goes
    
    __status = 'queued' # queued, setting up, running, ended
    __result = None # None, failed setup, failed, crashed, crashed server, timed out, canceled, succeeded
    __resultInfo = {
        'failed setup':   {'label':'SETUP  '},
        'failed':         {'label':'FAILED '},
        'crashed':        {'label':'CRASHED'},
        'crashed server': {'label':'SERVER '},
        'timed out':      {'label':'TIMEOUT'},
        'canceled':       {'label':'CANCEL '},
        'succeeded':      {'label':'SUCESS '}
    }
    __testProcess = None
    __consoleFile = None
    
    setupStartTime = None
    setupDuration = 0
    testStartTime = None
    testDuration = 0
    
    envVariablesToSet = None
    envVariables = None # set at run time
    
    consoleFile = sys.stdout
    returncode = None
    errorMessage = None # single-line description
    errorDetails = None # multi-line details, e.g.: tracebacks
    
    def __init__(self, name, path, driverLang=None, timeout=None):
        self.name = name
        self.path = path
        self.driverLang = driverLang
        
        self.envVariablesToSet = {}
        
        if timeout is not None:
            try:
                self.timeout = float(timeout)
            except Exception:
                raise ValueError('timeout must be a numeric value, got: %s' % str(timeout))
    
    @property
    def interpreter_path(self):
        if self.driverLang:
            return self.driverLang.interpreter_path
        else:
            return None
    
    @property
    def command(self):
        if self.path is None:
            raise RuntimeError('%s %s has no path defined when `command` was called' % (self.__class__.__name__, self.name))
        
        command = []
        if self.interpreter_path is not None:
            if not os.access(self.interpreter_path, os.X_OK):
                raise RuntimeError('interpreter is not valid/runnable: %s' % str(self.interpreter_path))
            command += [self.interpreter_path]
        command += [str(self.path)]
        return command
    
    @property
    def status(self):
        return self.__status
    
    @property
    def result(self):
        if self.__status != 'ended':
            return None
        else:
            return self.__result
    
    @result.setter
    def result(self, value):
        '''Set the result to one of the following: failed, failed setup, crashed server, crashed, timed out, canceled, succeeded'''
        
        if value not in ('failed', 'failed setup', 'crashed server', 'crashed', 'timed out', 'canceled', 'succeeded'):
            raise ValueError('result can not be "%s", rather it must be one of: failed, failed setup, crashed server, crashed, canceled, succeeded' % str(value))
        
        self.__result = value
    
    @property
    def resultLabel(self):
        resultInfo = self.__resultInfo.get(self.result, None)
        if resultInfo is None:
            return None
        else:
            return resultInfo['label']
    
    @property
    def duration(self):
        return (self.setupDuration or 0) + (self.testDuration or 0)
    
    @property
    def console_output(self):
        errorReturn = self.errorMessage or ''
        if self.errorDetails:
            if '\n' in self.errorDetails:
                errorReturn += '\n'
            errorReturn += self.errorDetails
            if '\n' in self.errorDetails:
                errorReturn += '\n'
        
        if self.consoleFile is not sys.stdout:
            try:
                with open(self.consoleFile, 'r') as consoleFile:
                    errorReturn += ("\n" if errorReturn else '') + consoleFile.read()
            except Exception: pass
        return errorReturn
    
    def queueTest(self):
        '''Called when the test is put (back) into the queue'''
        
        self.__status = 'queued'
        self.setupStartTime = None
        self.setupDuration = 0
        self.testStartTime = None
        self.testDuration = 0
    
    def setupTest(self):
        '''Do any pre-run setup that needs to be done for this test'''
        
        if self.__status != 'queued':
            raise RuntimeError('Something went wrong, we were going from "%s" to "setting up"' % self.__status)
        
        self.__status = 'setting up'
        self.setupStartTime = time.time()
        
        try:
            self.subclassSetup()
        finally:
            self.setupDuration = time.time() - self.setupStartTime
    
    def subclassSetup(self):
        pass
    
    def startTest(self, server=None):
        '''Run the test'''
        
        if self.__status != 'setting up':
            raise RuntimeError('Something went wrong, we were going from "%s" to "running"' % self.__status)
        
        if not isinstance(server, Server):
            raise RuntimeError('startTest requires a server instance, got: %s' % str(server))
        
        self.__status = 'running'
        self.testStartTime = time.time()
        
        # -- collect ENV variables
        
        envVariables = os.environ.copy()
        
        # - server info
        envVariables.update({
            'RDB_DRIVER_PORT':str(server.driver_port),
            'RDB_SERVER_HOST':server.host,
            'TEST_RESULT_DIR':self.resultDir
        })
        
        # - driver info
        if self.driverLang:
            for variableName, value in self.driverLang.envVariablesToSet.items():
                if value:
                    envVariables[variableName] = value
                elif variableName in envVariables:
                    del envVariables[variableName]
        elif 'INTERPRETER_PATH' in envVariables:
            del envVariables['INTERPRETER_PATH']
        
        # - test info
        for variableName, value in self.envVariablesToSet.items():
            if value:
                envVariables[variableName] = value
            elif variableName in envVariables:
                del envVariables[variableName]
        
        # - stow the env variables used
        self.envVariables = envVariables
        
        # -- start the test process
        
        with testLock:
            self.__consoleFile = self.consoleFile if self.consoleFile is sys.stdout else open(self.consoleFile, 'w+')
            self.__testProcess = subprocess.Popen(self.command, stdin=devNull, stdout=self.__consoleFile, stderr=subprocess.STDOUT, preexec_fn=os.setpgrp, cwd=self.resultDir, env=envVariables, close_fds=True)
    
    def check(self):
        '''Check on a running test, reply with True if running, False if ended'''
        
        if self.__status != 'running':
            raise RuntimeError('checkTest called on a test that was %s' % self.__status)
        
        # --
        
        if self.__testProcess.poll() is None:
            return False
        
        elif self.__testProcess.returncode == 0:
            self.result = 'succeeded'
            self.returncode = self.__testProcess.returncode
        
        elif self.__testProcess.returncode < 0:
            self.result = 'crashed'
            self.returncode = self.__testProcess.returncode
        
        else:
            self.result = 'failed'
            self.returncode = self.__testProcess.returncode
        
        # -- set the end markers and cleanup
        
        self.testDuration = time.time() - self.testStartTime
        self.cleanupTest()
        
        # -- return True to signal it has ended
        
        return True
    
    def endTest(self, result=None, errorMessage=None, errorDetails=None):
        '''Interrupts a running test'''
        
        if not self.__status in ('running', 'setting up') and result != 'canceled':
            raise RuntimeError('endTest called on a test that was %s' % self.__status)
        
        self.errorMessage = errorMessage
        self.errorDetails = errorDetails
        
        # -- set the end markers and cleanup
        
        if self.__status == 'setting up':
            self.setupDuration = time.time() - self.setupStartTime
            self.result = result or 'failed setup'
        elif self.__status == 'running':
            self.testDuration = time.time() - self.testStartTime
            self.result = result or 'failed'
        else:
            self.result = result
        
        self.cleanupTest()
        
        if result == 'timed out':
            self.returncode = TIMED_OUT_EXIT_CODE
        elif result == 'canceled':
            self.returncode = CANCELED_EXIT_CODE
        else:
            self.returncode = None
    
    def cleanupTest(self):
        '''Perform after-test cleanup'''
        
        # -- kill everything in the process group
        
        if self.__testProcess is not None and self.__testProcess.pid is not None:
            utils.kill_process_group(self.__testProcess)
        
        # -- close the output file
        
        if self.__consoleFile and self.__consoleFile is not sys.stdout:
            self.__consoleFile.close()
        
        # -- toss the subprocess
        
        self.__testProcess = None
        
        # -- set the status to ended
        
        self.__status = 'ended'

class YamlTest(Test):
    buildFolder = os.path.join(os.path.realpath(os.path.dirname(__file__)), 'build')
    srcPath = None
    shards = None
    
    def __init__(self, name, path, driverLang=None, timeout=None, shards=1):
        self.srcPath = path
        self.shards = shards
        super(YamlTest, self).__init__(name, None, driverLang=driverLang, timeout=timeout)
        if print_debug:
            self.envVariablesToSet.update({'VERBOSE':'true'})
    
    @property
    def interpreter_path(self):
        return self.driverLang.interpreter_path
    
    def subclassSetup(self):
        '''Build test executable'''
        
        # -- ensure build folder exists
        
        try:
            os.makedirs(self.buildFolder)
        except OSError as e:
            if e.errno != 17:
                raise
        
        # -- build test file
        
        self.path = os.path.join(self.buildFolder, self.name.replace('/', '.'))
        
        TestGroup.buildYamlTest(testName=self.name, sourceFile=self.srcPath, lang=self.driverLang, outputPath=self.path, shards=self.shards)

class MochaTest(Test):
    '''JavaScript tests run using mocha and mocha.runner'''
    
    mochaPath = distutils.spawn.find_executable('mocha')
    
    def __init__(self, name, path, driverLang=None, timeout=None, shards=1):
        assert self.mochaPath is not None, 'Mocha does not seem to be installed on this system'
        assert isinstance(driverLang, JsLang), 'Mocha tests only support JavaScript'
        super(MochaTest, self).__init__(name, path, driverLang=driverLang, timeout=timeout)
        
        self.envVariablesToSet.update({'BLUEBIRD_DEBUG':'1'})
    
    @property
    def interpreter_path(self):
        return self.driverLang.interpreter_path
    
    @property
    def command(self):
        if self.path is None:
            raise RuntimeError('%s %s has no path defined when `command` was called' % (self.__class__.__name__, self.name))
        mochaOptions = ['--reporter', 'spec']
        if not utils.supportsTerminalColors():
            mochaOptions += ['--no-colors'] # handle visionmedia/mocha#1304
        return [self.interpreter_path, self.mochaPath] + mochaOptions + [str(self.path)]

class HttpbinTest(Test):
    '''Test that uses a httpbin target'''
    
    targetServer = None # shared
    mochaPath = distutils.spawn.find_executable('mocha')
    
    def __init__(self, name, path, driverLang=None, timeout=None, shards=1):
        self.shards = shards
        super(HttpbinTest, self).__init__(name, path, driverLang=driverLang, timeout=timeout)
        if print_debug:
            self.envVariablesToSet.update({'VERBOSE':'true'})
    
    @property
    def interpreter_path(self):
        return self.driverLang.interpreter_path
    
    @property
    def command(self):
        if self.path is None:
            raise RuntimeError('%s %s has no path defined when `command` was called' % (self.__class__.__name__, self.name))
        
        if self.driverLang and self.driverLang.language_name == 'JavaScript':
            mochaOptions = ['--reporter', 'spec']
            if not utils.supportsTerminalColors():
                mochaOptions += ['--no-colors'] # handle visionmedia/mocha#1304
            return [self.interpreter_path, self.mochaPath] + mochaOptions + [str(self.path)]
        else:
            return super(HttpbinTest, self).command
    
    def subclassSetup(self):
        '''Start the httpbin server'''
        if self.targetServer is None:
            self.__class__.targetServer = http_support.HttpTargetServer()
        self.envVariablesToSet.update({
            'HTTPBIN_HOST':             'localhost',
            'HTTPBIN_HTTPBIN_PORT':     str(self.targetServer.httpbinPort),
            'HTTPBIN_HTTPCONTENT_PORT': str(self.targetServer.httpPort),
            'HTTPBIN_HTTPS_PORT':       str(self.targetServer.sslPort)
        })

def getTestList(rootPath, allowedLanguages, testFilters=None, shards=0):
    '''Get a list of Test objects for tests found in the given folder'''
    
    # -- input validation
    
    # rootPath
    
    if rootPath is None:
        raise ValueError('getTestList got None for rootPath')
    if not os.path.isdir(str(rootPath)):
        raise ValueError('getTestList got a non-directory as rootpath: %s' % str(rootPath))
    rootPath = os.path.realpath(rootPath)
    
    # testFilters
    
    if testFilters is not None:
        if not hasattr(testFilters, '__iter__'):
            testFilters = [testFilters]
        for testFilter in testFilters:
            if not hasattr(testFilter, 'match'):
                raise ValueError('getTestList got a non-regex value as a filter: %s' % str(testFilter))
    
    # allowedLanguages
    
    if allowedLanguages is None:
        raise ValueError('getTestList requires one or more languages to look for')
    
    if not hasattr(allowedLanguages, '__iter__'):
        allowedLanguages = [allowedLanguages]
    
    for language in allowedLanguages:
        if not isinstance(language, SrcLang):
            raise ValueError('getTestList got a language that it does not know how to process: %s' % language)
        
    # -- find items in the directory
    
    foundTests = []
    langSetsCache = { # cache of: testLangExtensions -> set(langauges)
        'mocha': [x for x in interpreters['js'] if x in allowedLanguages],
        'yaml':  allowedLanguages
    } 
    testNameRegex = re.compile(r'^(?P<name>\w+)(\.(?P<langs>\+?([a-zA-Z]+(\d+(_\d+)*)?)(_one)?\+?(,\+?([a-zA-Z]+(\d+(_\d+)*)?)(_one)?\+?)*))?\.(?P<ext>\w+)$', re.IGNORECASE)
    langExtRegex = re.compile(r'^(?P<pre>\+)?(?P<lang>[a-zA-Z]+)(?P<version>\d+(_\d+)*)?(?P<single>_one)?(?P<post>\+)?$', re.IGNORECASE)
    
    for root, dirs, files in os.walk(rootPath):
        
        groupName = os.path.relpath(root, rootPath)
        if groupName in ('.', './'):
            groupName = ''
        if groupName == 'src' or groupName.startswith('src/'):
            groupName = 'polyglot' + groupName.lstrip('src')
        
        for fileName, testName, testLangExtensions, extension in ((xa, ) + y.group('name', 'langs', 'ext') for xa, y in ((x, testNameRegex.match(x)) for x in files) if y is not None):
            testName = os.path.join(groupName, testName)
            filePath = os.path.join(root, fileName)
            
            if extension not in testExtensions:
                continue
            
            # - process testLangExtensions
            
            testLanguages = set() if testLangExtensions else None
            if testLangExtensions and testLangExtensions in langSetsCache:
                testLanguages = langSetsCache[testLangExtensions]
            elif testLangExtensions:
                for langExt in testLangExtensions.split(','):
                    res = langExtRegex.match(langExt)
                    if not res:
                        warnings.warn('File somehow has a bad language extension (%s): %s' % (langExt, filePath)) # should not be possible with the regex
                        break
                    pre, lang, version, single, post = res.group('pre', 'lang', 'version', 'single', 'post')
                    
                    if lang not in interpreters:
                        warnings.warn('File has unknown interpreter `%s`: %s' % (lang, filePath))
                        break
                    
                    if pre and post:
                        warnings.warn('File uses both prefix and suffix `+` symbols in the language portion: %s' % filePath)
                        break
                    elif pre:
                        preVersion = distutils.version.LooseVersion(version.replace('_', '.').lower() + '.99999')
                        for language in (x for x in interpreters[lang] if x in allowedLanguages):
                            if language.interpreter_version <= preVersion:
                                testLanguages.add(language)
                                if single:
                                    break
                    elif post:
                        postVersion = distutils.version.LooseVersion(version.replace('_', '.').lower())
                        for language in (x for x in interpreters[lang] if x in allowedLanguages):
                            if language.interpreter_version >= postVersion:
                                testLanguages.add(language)
                                if single:
                                    break
                    else:
                        # all of the versions this works for
                        testLanguages = [x for x in interpreters[lang] if x in allowedLanguages] or None
                        if single and testLanguages:
                            testLanguages = [testLanguages[0]]
                        testLanguages = set(testLanguages or [])
                langSetsCache[testLangExtensions] = testLanguages
            
            elif extension in ('mocha', 'yaml'):
                testLanguages = langSetsCache[extension]
            else:
                testLanguages = set([None])
            
            # -- create tests for all language versions that pass the filters
            
            for language in testLanguages or []:
                subTestName = '%s.%s' % (testName, language.test_types[0]) if language else testName
                
                # - run the filters
                if testFilters is not None and len(testFilters) > 0:
                    for testFilter in testFilters:
                        if testFilter.match(subTestName) is not None:
                            break
                    else:
                        continue
                
                # - add the test
                if extension == 'yaml':
                    foundTests.append(YamlTest(name=subTestName, path=filePath, driverLang=language, shards=shards))
                elif extension == 'mocha':
                    foundTests.append(MochaTest(name=subTestName, path=filePath, driverLang=language, shards=shards))
                elif extension == 'httpbin':
                    foundTests.append(HttpbinTest(name=subTestName, path=filePath, driverLang=language, shards=shards))
                else:
                    foundTests.append(Test(name=subTestName, path=filePath, driverLang=language))
    
    # -- return a sorted list
    
    foundTests.sort(key=lambda x: (x.name, x.type))
    return foundTests

class WorkerThread(threading.Thread):
    '''Serially runs Test objects from its workQueue, managing a server instance and optionally returning (status-test, test) tuples'''
    
    daemon = True
    
    workQueue = None
    outputQueue = None
    
    outputDir = None
    scratchDir = None
    writeToConsole = None
    
    server = None
    existingDBsToTables = None # dict of arrays of db_name => table_name already on this server
    
    def __init__(self, workQueue, outputQueue, outputDir, scratchDir, externalServer=None, writeToConsole=False):
        if workQueue is None or not isinstance(workQueue, Queue.Queue):
            raise ValueError('workQueue must be a Queue object, got: %r' % workQueue)
        self.workQueue = workQueue
        
        if outputQueue is not None and not isinstance(outputQueue, Queue.Queue):
            raise ValueError('outputQueue must be a Queue object, got: %r' % outputQueue)
        self.outputQueue = outputQueue
        
        if not os.path.isdir(outputDir):
            raise ValueError('outputDir must be an existing directory, got: %s' % outputDir)
        self.outputDir = outputDir
        
        if not os.path.isdir(scratchDir):
            raise ValueError('scratchDir must be an existing directory, got: %s' % scratchDir)
        self.scratchDir = os.path.realpath(scratchDir)
        
        if externalServer is not None:
            if not isinstance(externalServer, ExternalServer):
                raise ValueError('externalServer must be an instance of ExternalServer, got: %r' % externalServer)
            self.server = externalServer
        
        self.writeToConsole = writeToConsole is True
        
        super(WorkerThread, self).__init__()
        self.start()
    
    def run(self):
        
        class CancelRunError(Exception):
            pass
        
        while cancelRun is False:
            test = None
            
            try:
                # -- get a test
                
                if cancelRun:
                    raise CancelRunError()
                
                try:
                    test = self.workQueue.get_nowait()
                except Queue.Empty:
                    break
                 
                # -- make sure we have a running server
                
                if self.server is None:
                    self.server = Server(scratchDir=self.scratchDir)
                try:
                    self.server.check()
                except Exception:
                    if isinstance(self.server, ExternalServer):
                        raise CancelRunError('External server no longer responding!')
                    else:
                        try:
                            self.server.stop()
                        except Exception: pass
                        self.server = Server(scratchDir=self.scratchDir)
                        self.server.check()
                            
                # -- ensure the server is clean
                    
                self.server.clean()
                
                # -- pre-process the test
                
                if cancelRun:
                    raise CancelRunError()
                
                if self.outputQueue:
                    self.outputQueue.put(('setting up', test, time.time()))
                try:
                    test.setupTest()
                except Exception as e:
                    debug(traceback.format_exc())
                    test.endTest(errorMessage='Failed setting up test %s' % str(test.name), errorDetails=traceback.format_exc())
                    self.outputQueue.put((test.status, test, time.time()))
                    continue
                
                # -- start the test
                
                if cancelRun:
                    raise CancelRunError()
                
                # - create test output folder
                test.resultDir = os.path.join(self.outputDir, test.name)
                if os.path.exists(test.resultDir):
                    # delete the old item
                    if os.path.isdir(test.resultDir):
                        shutil.rmtree(test.resultDir)
                    else:
                        os.unlink(test.resultDir)
                os.makedirs(test.resultDir)
                
                # - set consoleFile
                test.consoleFile = sys.stdout if self.writeToConsole else os.path.join(test.resultDir, 'console.txt')
                
                # - run the test
                if cancelRun:
                    raise CancelRunError()
                
                try:
                    self.outputQueue.put(('running', test, time.time()))
                    test.startTest(server=self.server)
                except Exception as e:
                    debug(traceback.format_exc())
                    test.endTest(errorMessage='Failed starting test', errorDetails=traceback.format_exc())
                    self.outputQueue.put((test.status, test, time.time()))
                    continue
                
                # -- monitor the running test
                
                restartServer = False
                copyServerContents = False
                deadline = time.time() + test.timeout
                    
                while time.time() < deadline:
                    if cancelRun:
                        raise CancelRunError()
                    try:
                        if test.check():
                            if test.returncode != 0:
                                restartServer = True
                            break
                        try:
                            self.server.check()
                        except Exception as e:
                            test.endTest(result='crashed server', errorMessage='Server crashed during testing')
                            restartServer = True
                            copyServerContents = True
                            break
                        time.sleep(.1)
                    except Exception as e:
                        debug(traceback.format_exc())
                        test.endTest(errorMessage='Exception while monitoring', errorDetails=traceback.format_exc())
                        self.outputQueue.put((test.status, test, time.time()))
                        break
                else:
                    # ToDo: sample running processes
                    test.endTest(result='timed out', errorMessage='Timed out after %d seconds' % test.timeout)
                
                # -- post-test
                
                # - check the server
                if test.result != 'crashed server':
                    try:
                        self.server.check()
                    except Exception as e:
                        if test.result == 'succeeded':
                            test.result = 'crashed server'
                        
                        restartServer = True
                        copyServerContents = True
                        
                        if test.errorMessage is None:
                            test.errorMessage = 'Server found crashed after test'
                        else:
                            test.errorMessage += ', and the server crashed while running'
                
                # - copy the server contents or just the server log
                if self.server.logfile_path and test.resultDir: # in case this is a local server, change after https://github.com/rethinkdb/rethinkdb/issues/4512
                    if copyServerContents:
                        shutil.copytree(self.server.data_path, os.path.join(test.resultDir, 'server'))
                        
                        # try to add the server log to the errorDetails
                        try:
                            if test.errorDetails is None:
                                test.errorDetails = '\nServer output:\n%s' % self.server.console_output
                            else:
                                test.errorDetails += '\n\nServer output:\n%s' % self.server.console_output
                        except IOError: pass # stdout will cause this
                        except Exception as e:
                            warnings.warn(str(e))
                    else:
                        shutil.copyfile(self.server.logfile_path, os.path.join(test.resultDir, 'server_log.txt'))
                
                # - write test metadata file
                with open(os.path.join(test.resultDir, 'metadata.json'), 'w+') as metadataFile:
                    command = ''
                    try:
                        command = test.command
                    except RuntimeError: pass
                    
                    metaData = {
                        'name': test.name,
                        'result': test.result,
                        'returncode': test.returncode,
                        'test duration sec': test.testDuration,
                        'test start time': datetime.datetime.fromtimestamp(test.testStartTime).isoformat(),
                        'setup duration sec': test.setupDuration,
                        'setup start time': datetime.datetime.fromtimestamp(test.setupStartTime).isoformat(),
                        'command': command,
                        'env variables': test.envVariables
                    }
                    json.dump(metaData, metadataFile, indent=1)
                
                # - return result
                self.outputQueue.put((test.status, test, time.time()))
                self.workQueue.task_done() # not really important
                
                test = None
        
            except CancelRunError:
                if test:
                    test.endTest(result='canceled')
                    self.outputQueue.put((test.status, test, time.time()))
                    self.workQueue.task_done() # not really important
                    break
            
            except Exception as e:
                debug(traceback.format_exc())
                warnings.warn("Worker failure: %s" % unicode(e))
                restartServer = True
                if test:
                    test.endTest(result='failed', errorMessage='Exception while running `%s`: %s' % (str(test.name), str(e)), errorDetails=traceback.format_exc())
            
            # -- shutdown the server if necessary
            if restartServer:
                self.server.stop() # just cleans for external servers
                if not isinstance(self.server, ExternalServer):
                    try:
                        self.server.stop()
                    except Exception: pass
                    self.server = None
        
        # -- shutdown server before worker exit
        try:
            if self.server:
                self.server.stop() # just cleans for external servers
        except Exception as e:
            debug(traceback.format_exc())
            sys.stdout.write('Server issue while stoping worker: %s' % str(e))
        self.server = None

# ==== Main

def main():
    global print_debug
    global r
    
    testFilters = []
    testList = []
    
    outputDir = None  # folder where the final results go
    scratchDirPath = None # used for temporary files like database files
    
    rethinkdb_exe_path = None
    externalServer = None
    
    # -- parse command line options
    
    usage = '''
usage: %prog [options] [patterns]

Tests by default use the drivers from the appropriate folder in this
repository. To change this set the appropriate environmental variable
from this list: JAVASCRIPT_DRIVER, PYTHON_DRIVER, RUBY_DRIVER. Or use
`--INSTALLED--` to use the system-installed version.
'''.strip()
    
    parser = optparse.OptionParser(usage=usage)
    
    parser.add_option('-l', '--list', dest='list_mode', default=False, action='store_true', help='list the tests and exit')
    
    parser.add_option('-b', '--server-binary', dest='rethinkdb_exe_path', default=None, help='path to RethinkDB server binary to test')
    parser.add_option('-o', '--output-dir', dest='output_dir', default=None, help='parent directory for test output folder')
    parser.add_option(      '--scratch-dir', dest='scratch_dir', default=None, help='parent directory for a temporary scratch folder')
    parser.add_option(      '--clean', dest='clean_output_dir', default=False, action='store_true', help='clean the output directory before running')
    
    parser.add_option('-i', '--interpreter', dest='languages', action='callback', callback=check_language, choices=list(interpreters.keys()), type='choice', default=None, help='the language to test')
    parser.add_option('-s', '--shards', dest='shards', type='int', default=1, help='number of shards to run (default 1)')
    
    parser.add_option('-d', '--driver-port', dest='driver_port', default=None, help='driver port of an already-running rethinkdb instance')
    parser.add_option('-t', '--table', dest='table', default=None, type='string', help='name of pre-existing table to run queries against')
    
    parser.add_option(      '--debug', dest='debug', default=False, action='store_true', help='show %s debug output' % os.path.basename(__file__))
    parser.add_option('-v', '--verbose', dest='verbose', default=False, action='store_true', help='show test output while running (disables -j/--jobs')
    
    parser.add_option('-j', '--jobs', dest='workerThreads', default=None, type='int', help='tests to run simultaneously (default 2)')
    
    options, args = parser.parse_args()
    
    # - options validation
    
    # debug
    print_debug = options.debug
    
    # workerThreads
    if options.workerThreads is None:
        if options.table or options.verbose or options.driver_port:
            options.workerThreads = 1
        else:
            options.workerThreads = 2
    
    elif options.verbose is True:
        warnings.warn('-v/--verbose disables running more than one test in parallel')
        options.workerThreads = 1
    
    elif options.workerThreads < 1:
        parser.error('-j/--jobs value must be 1 or greater')
    
    # outputDir
    if options.output_dir is None:
        # default to a hidden directory that is deleted at exit
        outputDir = tempfile.mkdtemp(prefix='.rethinkdbTestTemp-', dir=os.getcwd())
        utils.cleanupPathAtExit(outputDir)
    else:
        if os.path.exists(options.output_dir):
            if not os.path.isdir(options.output_dir):
                parser.error('-o/--ouput-dir already exists, and is not a directory')
        else:
            try:
                os.makedirs(options.output_dir)
            except Exception as e:
                parser.error('Unable to create output directory: %s' % str(e))
        outputDir = options.output_dir
    
    # scratchDirPath
    if options.scratch_dir is None:
        # default to a hidden subdir of outputDir
        options.scratch_dir = outputDir
    scratchDirPath = tempfile.mkdtemp(prefix='.scratch-', dir=options.scratch_dir)
    # auto-clean this folder
    utils.cleanupPathAtExit(scratchDirPath)
    
    # - add filters
    
    for newFilter in args:
        try:
            testFilters.append(re.compile(newFilter))
        except Exception:
            parser.error('Invalid filter (regex) entry: %s' % newFilter)
    
    # - pull in environmental variables
    
    if len(testFilters) == 0:
        if os.getenv('RQL_TEST'):
            try:
                testFilters.append(re.compile(os.getenv('RQL_TEST')))
            except re.error:
                parser.error("'Invalid filter from ENV: %s" % os.getenv('RQL_TEST'))
        if os.getenv('TEST'):
            try:
                testFilters.append(re.compile(os.getenv('TEST')))
            except re.error:
                parser.error("'Invalid filter from ENV: %s" % os.getenv('TEST'))
    
    # - default options
    
    if options.languages is None:
        options.languages = []
        for languageGroup in (interpreters['js'], interpreters['py'], interpreters['rb']):
            for language in languageGroup:
                try:
                    language.interpreter_path # raises exception if not present/found
                    options.languages.append(language)
                    break
                except Exception: pass
        if len(options.languages) == 0:
            parser.error('Unable to find interpreters for any of the default languages')
    
    # -- get the list of tests
    
    testList = getTestList(os.path.realpath(os.path.dirname(__file__)), allowedLanguages=options.languages, testFilters=testFilters, shards=options.shards)
    
    # -- clean output dir if requested
    
    if options.clean_output_dir is True:
        try:
            for item in os.listdir(outputDir):
                item = os.path.join(outputDir, item)
                if os.path.samefile(item, scratchDirPath):
                    continue
                if os.path.isdir(item):
                    shutil.rmtree(item)
                else:
                    os.unlink(item)
        except Exception as err:
            parser.error('Unable to clean output directory: %s' % str(err))
    
    # -- short-circuit if there are no tests
    
    if len(testList) == 0:
        languagesString = 'for the language: '
        if len(options.languages) > 1:
            languagesString = 'for the languages: '
        languagesString += ', '.join([lang.display_name for lang in options.languages])
        if len(args) == 1:
            sys.stderr.write('There are no tests that match the filter: %s %s\n' % (args[0], languagesString))
        elif len(args) > 1:
            sys.stderr.write('There are no tests that match any of the filters: %s %s\n' % (args, languagesString))
        else:
            sys.stderr.write('There are no tests %s\n' % languagesString)
        sys.exit()
    
    # -- print list if requested
    
    if options.list_mode is True:
        for test in testList:
            print(test.name)
        sys.exit()
    
    # -- reduce the languages to only ones we have tests selected for
    
    for language in options.languages[:]:
        for test in testList:
            if test.driverLang == language:
                break
        else:
            options.languages.remove(language)
            sys.stderr.write('Warning: did not find any tests for %s\n' % language.display_name)
    
    # -- check the rethinkdb_exe_path
    
    if options.rethinkdb_exe_path is None:
        try:
            rethinkdb_exe_path = utils.find_rethinkdb_executable()
        except test_exceptions.TestingFrameworkException as e:
            sys.exit(str(e))
    else:
        rethinkdb_exe_path = options.rethinkdb_exe_path
    
    if not os.access(rethinkdb_exe_path, os.X_OK):
        sys.exit('Error: selected RethinkDB server binary does not look valid: %s' % rethinkdb_exe_path)
    
    # -- set ENV settings for child processes
    
    os.environ['RDB_EXE_PATH'] = rethinkdb_exe_path
    
    if options.table:
        os.environ['TEST_DB_AND_TABLE_NAME'] = options.table
    
    # -- make sure the drivers are built
    
    for language in options.languages:
        if not language.driver_info['sourcePath']:
            continue
        
        outputFile = tempfile.NamedTemporaryFile(mode='w+')
        notificationDeadline = time.time() + 3
        makeProcess = subprocess.Popen(['make', '-C', language.driver_info['sourcePath']], stdin=devNull, stdout=outputFile, stderr=subprocess.STDOUT, close_fds=True)
        while makeProcess.poll() is None and time.time() < notificationDeadline:
            time.sleep(.1)
        if time.time() > notificationDeadline:
            print('Building the %s drivers. This make take a few moments.' % language.display_name)
        if makeProcess.wait() != 0:
            sys.stderr.write('Error making %s driver. Make output follows:\n\n' % os.path.basename(__file__))
            outputFile.seek(0)
            print(outputFile.read().decode('utf-8'))
            sys.exit(1)
        
        os.environ[language.driver_info['envName'] + '_SRC'] = language.driver_info['sourcePath']
    
    r = utils.import_python_driver()
    
    # driver_port
    if options.driver_port is not None:
        host = 'localhost'
        port = options.driver_port
        if ':' in options.driver_port:
            host, port = options.driver_port.split(':', 1)
        try:
            port = int(port)
            assert port > 0
        except ValueError:
            parser.error('-d/--driver-port must be a positive integer or a hostname:integer pair. Got: %s' % options.driver_port)
        
        try:
            externalServer = ExternalServer(port, host=host)
        except Exception as e:
            debug(traceback.format_exc())
            parser.error('Error connecting to the provided server: %s' % str(e))
        if options.shards != 1:
            parser.error('-d/--driver-port can not be used with -s/--shards')
    elif options.table:
        parser.error('If you specify -t/--table, you must also specify -d/--driver-port')
    
    if options.table and options.table.count(".") != 1:
        parser.error('Parameter to -t/--table should be of the form db.table')
    
    # -- print pre-testing info
    
    print('Using rethinkdb binary %s' % rethinkdb_exe_path)
    for thisLanguage in options.languages:
        print('\t%s interpreter: %s, driver: %s' % (thisLanguage.display_name, thisLanguage.interpreter_path, thisLanguage.driver_info['driverPath']))
        if outputDir not in utils.pathsToClean:
            print('\toutput folder: %s' % outputDir)
    
    startTime = time.time()
    
    # -- add tests to queues
    
    testQueue = Queue.Queue()
    outputQueue = Queue.Queue()
    
    for test in testList:
        testQueue.put(test)
    
    # -- start the worker threads
    
    workerThreads = [WorkerThread(
        workQueue=testQueue, outputQueue=outputQueue,
        outputDir=outputDir, scratchDir=scratchDirPath,
        externalServer=externalServer, writeToConsole=options.verbose
    ) for x in range(options.workerThreads)]
    
    # -- monitor progress
    
    failedTests = []
    
    preparingTests = 0
    runningTests = 0
    waitingTests = len(testList)
    canceledTests = 0
    passedTests = 0
    
    while True:
        
        # - print all status messages
        while True:
            message = None
            test = None
            try:
                message, test, eventTime = outputQueue.get_nowait()
            except Queue.Empty:
                break
            
            timeString = 'T+ %.1f sec' % (eventTime - startTime)
            
            if message == 'queued':
                pass
            elif message == 'setting up':
                sys.stdout.write('== Starting: %s (%s)\n' % (test.name, timeString))
                preparingTests -= 1
                preparingTests += 1
            elif message == 'running':
                #sys.stdout.write('== Running: %s (%s)\n' % (test.name, timeString))
                waitingTests -= 1
                runningTests += 1
            elif message == 'ended':
                runningTests -= 1
                
                durationString = None
                if test.testDuration:
                    durationString = '%.1f sec + %.1f sec setup' % (test.testDuration, test.setupDuration)
                else:
                    durationString = '%.1f sec setup' % test.setupDuration
                                
                if test.result is None:
                    warnings.warn('Got None for test.result for %r, this should not happen' % test.name)
                    continue
                
                if test.result == 'succeeded':
                    sys.stdout.write('== Passed: %s in %s (%s)\n' % (test.name, durationString, timeString))
                    passedTests += 1
                
                elif test.result == 'canceled':
                    sys.stdout.write('== Canceled: %s after %s (%s)\n' % (test.name, durationString, timeString))
                    canceledTests += 1
                
                else:
                    failedTests.append(test)
                    
                    extraMessage = ''
                    if test.result == 'failed':
                        extraMessage = ' with exit code %s' % test.returncode
                    elif test.result == 'crashed':
                        extraMessage = ' with signal %s' % abs(test.returncode)
                    elif test.result == 'timed out':
                        extraMessage = ' after %d seconds' % test.duration
                    
                    consoleOutput = test.console_output.strip()
                    if consoleOutput:
                        sys.stderr.write('>>> %s %s%s after %s (%s)\n\n' % (test.result.capitalize(), test.name, extraMessage, durationString, timeString))
                        sys.stderr.write(consoleOutput)
                        sys.stderr.write('\n<<< end %s: %s\n' % (test.result.capitalize(), test.name))
                    else:
                        sys.stderr.write('>>> %s %s%s after %s (%s) <<<\n' % (test.result.capitalize(), test.name, extraMessage, durationString, timeString))
            sys.stderr.flush()
            sys.stdout.flush()
        
        # - end if there are no worker threads
        if len(workerThreads) == 0:
            break
        
        # - reap dead worker threads
        for worker in copy.copy(workerThreads):
            if not worker.isAlive():
                worker.join(.01) # should never timeout, but just in case
                workerThreads.remove(worker)
        
        # -
        time.sleep(.1)
    
    # -- check that all tests have a finished status
    
    if cancelRun is False:
        for test in testList:
            if test.status != 'ended':
                raise Warning('Test was in state "%s" after run: %s' % (test.status, test.name))
    
    # -- report final results
    
    failedTestLines = ['\t%s %s' % (test.resultLabel, test.name) for test in sorted(failedTests, key=lambda x: x.name)]
    
    if cancelRun is True:
        print('\n== Canceled after %.2f seconds with %d test%s running. %s test%s passed, %s test%s failed, and %s test%s remaining%s' % (
            time.time() - startTime,
            canceledTests, 's' if canceledTests != 1 else '',
            passedTests, 's' if passedTests != 1 else '',
            len(failedTestLines), 's' if len(failedTestLines) != 1 else '',
            waitingTests, 's' if waitingTests != 1 else '',
            ('\n' + '\n'.join(failedTestLines) + '\n\n' if len(failedTestLines) > 0 else '')
        ))
        sys.exit(3)
    elif len(failedTests) == 0:
        testNumberMessage = 'the 1 test'
        if len(testList) > 1:
            testNumberMessage = 'all %s tests' % len(testList)
        print('\n== Successfully passed %s in %.2f seconds!' % (testNumberMessage, time.time() - startTime))
        sys.exit(0)
    else:
        plural = 's' if len(failedTests) > 1 else ''
        print('\n== Failed %d test%s (of %d) in %.2f seconds!\n%s\n\n' % (len(failedTests), plural, len(testList), time.time() - startTime, '\n'.join(failedTestLines)))
        sys.exit(1)
        
if __name__ == '__main__':
    main()
